package com.gxljc.bear.crawler.itaogao.jjckb;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;
import com.gxljc.bear.crawler.util.DateUtil;
import com.gxljc.bear.crawler.util.JsoupUtil;
import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 经济参考报 爬虫 Created by Administrator -> haitao on 2016/9/13.
 */
public class JjckbDetailCrawler extends NewsBaseCrawler {

	private static final long serialVersionUID = 600742470078941777L;
	private static final Logger LOG = Logger.getLogger(JjckbDetailCrawler.class);
	private static final String EDITION_URL_LIST = "http://dz.jjckb.cn/www/pages/webpage2009/html/%s/node_%s.htm";
	private static final String PAGE_LIST = "http://dz.jjckb.cn/www/pages/webpage2009/html/%s/%s";
	private static final String DEFAULT_DATE_FORMAT = "yyyy-MM/dd";

	public JjckbDetailCrawler() {
		super("t_newspaper_jjckb", DolphinCrawlerConsts.MongoDBName.ITAOGAO.getName());
	}

	@SuppressWarnings("unchecked")
	@Override
	protected List<String> genSeeds() {
		LOG.info("--- genSeed of Jjckb ----");
		List<String> seeds = new ArrayList<>();
		try {
			for (String date : genDatesTillNow(DEFAULT_DATE_FORMAT)) {
				// fetch url list by parser
				int index = 1;
				while (true) {
					String url = String.format(EDITION_URL_LIST, date, "" + (++index));
					if (fetch(url, proxy) == null) {
						break;
					} else {
						String nodeResult = "";
						Pair<String, byte[]> pair2 = fetResult(url, proxy);
						if (pair2 == null) {
							continue;
						}
						nodeResult = pair2.first;
						if (nodeResult != null) {
							JSONObject json2 = (JSONObject) JSONObject.parse(nodeResult);
							if (json2 != null) {
								List<String> pageUrl = (List<String>) json2.get("page_url");
								if (pageUrl != null) {
									seeds.addAll(completeUrl(date, pageUrl));

								}
							}
						}

					}

				}

			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	private List<String> completeUrl(String date, List<String> pageUrl) {
		List<String> list = new LinkedList<>();
		for (String s : pageUrl) {
			if (s.startsWith("content")) {
				list.add(String.format(PAGE_LIST, date, s));
			}
		}
		return list;
	}

	@SuppressWarnings("unchecked")
	@Override
	protected List<String> genNewSeeds() {
		String date = DateUtil.dateFormat(new Date(), "yyyy-MM/dd");
		LOG.info("--- genNewSeed of Jjckb ----");
		List<String> seeds = new ArrayList<>();
		try {

			// fetch url list by parser
			int index = 1;
			while (true) {
				String url = String.format(EDITION_URL_LIST, date, "" + (++index));
				if (fetch(url, proxy) == null) {
					break;
				} else {
					String nodeResult = "";
					Pair<String, byte[]> pair2 = fetResult(url, proxy);
					if (pair2 == null) {
						continue;
					}
					nodeResult = pair2.first;
					if (nodeResult != null) {
						JSONObject json2 = (JSONObject) JSONObject.parse(nodeResult);
						if (json2 != null) {
							List<String> pageUrl = (List<String>) json2.get("page_url");
							if (pageUrl != null) {
								seeds.addAll(completeUrl(date, pageUrl));

							}
						}
					}

				}
			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	@Override
	protected Pair<NewsBaseTable, byte[]> genTable(String url) {
		NewsBaseTable table = new NewsBaseTable();
		// parse page by Jsoup
		byte[] html = null;
		try {
			html = fetch(url, proxy);
		} catch (Exception e) {
			e.printStackTrace();
		}
		if (html != null) {
			String htmlPage = new String(html);
			table = extractTable(htmlPage);
		}
		table.setPublishDate(extractDate(url));
		// set default value
		table.setId(StringUtil.genMD5Val(url));
		table.setCrawlUrl(url);
		table.setCrawlDate(new Date());
		table.setCrawlFlag(1);
		return new Pair<>(table, html);

	}

	private NewsBaseTable extractTable(String html) {
		NewsBaseTable table = new NewsBaseTable();
		List<String> titleList = JsoupUtil.extractList(html, ".hei16b");
		if (validateObject(titleList)) {
			table.setTitle(titleList.get(0));
		}
		List<String> boxList = JsoupUtil.extractList(html, ".black12");

		if (validateObject(boxList)) {
			table.setAuthor(extractAuthor(boxList.get(0)));
			table.setSource(extractByPrefix("来源：", boxList.get(0)));
		}
		List<String> paraList = JsoupUtil.extractList(html, ".black14 founder-content p");
		if (validateObject(paraList)) {
			List<String> nList = new LinkedList<>();
			nList.addAll(boxList);
			nList.addAll(paraList);
			//cjjkb正文无图片展示
			table.setContent(genContent(nList).replaceAll("&nbsp;", ""));
		}
		return table;
	}

	/**
	 * @param box
	 * @return [2016-09-08 &nbsp;&nbsp;&nbsp;&nbsp;□侯云龙 来源：经济参考报] ->
	 */

	private String extractAuthor(String box) {
		Pattern pattern = Pattern.compile("□(.*?) 来源");
		Matcher matcher = pattern.matcher(box);
		String name = "";
		while (matcher.find()) {
			name = matcher.group(1);
		}
		return name;
	}

}
